import java.util.regex.Pattern
import java.util.regex.Matcher
import groovy.io.FileType

import static groovy.io.FileType.FILES
import com.aliasi.util.Files
import de.fusionfactory.corpora.LeipzigCorpus

/**
 * Created by Markus Ackermann.
 * No rights reserved. 
 */
class PrepareCorpora {
    static final Pattern sentFilePattern =
        ~/(\p{Alpha}+)_(\p{Alpha}+)_(\d{4}(?:_\d{4})?)_(\d+[KM]{1})-sentences/

    static void handleSentenceFile(File inputSentenceFile, File outDir)
    throws IOException {
        def corpusDesc = LeipzigCorpus.fromFile inputSentenceFile

        FileInputStream fileIn = new FileInputStream(inputSentenceFile)
        InputStreamReader isReader = new InputStreamReader(fileIn, corpusDesc.encoding)
        BufferedReader bufReader = new BufferedReader(isReader);

        if( !outDir.exists()) { outDir.mkdirs() }
        File langOutDir = new File(outDir,corpusDesc.languageId);
        langOutDir.mkdir();
        File outFile = new File(langOutDir,"${corpusDesc.languageId}.txt");
        FileOutputStream fileOut = new FileOutputStream(outFile);
        OutputStreamWriter osWriter = new OutputStreamWriter(fileOut,"UTF-8");
        BufferedWriter bufWriter = new BufferedWriter(osWriter);

        println "\n" + corpusDesc.languageId
        println "reading from=$inputSentenceFile charset=$corpusDesc.encoding"
        println "writing to=$outFile charset=utf-8"

        long totalLength = 0L;
        String line;
        while ((line = bufReader.readLine()) != null) {
            if (line.length() == 0) continue;
            int index = line.indexOf("\t");
            String newline = line.substring(index+1);
            // System.out.println("line=" + line);
            // System.out.println("New line=" + newline);
            totalLength += newline.length();
            bufWriter.write(newline);
            bufWriter.write(" ");
        }
        println "total length=$totalLength"
        bufWriter.close();
        bufReader.close();
    }

    static void main(String[] args) {
        File inDir = new File(args[0]);
        File outDir = new File(args[1]);
        String[] languageDirNames = inDir.list();
        inDir.eachFile(FILES) { file ->
            def baseName = Files.baseName file
            if(baseName ==~ sentFilePattern) {
                PrepareCorpora.handleSentenceFile(file, outDir)
            }
        }
    }
}
